
# import necessary libraries
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import plotly
import copy
import cv2
import os


from sklearn import metrics as mt


# Add name of dataset to absolute path
data_directory = os.getcwd() 

# import csv file
df = pd.read_csv(data_directory + '\diamonds.csv')
df.head()


print(f"Number of observations in training data: {df.shape[0]}")
print(f"Number of features in training data: {df.shape[1]}\n\n")

print(df.dtypes)

Number of observations in training data: 53940
Number of features in training data: 10


carat      float64
cut         object
color       object
clarity     object
depth      float64
table      float64
price      float64
'x'        float64
'y'        float64
'z'        float64
dtype: object


df = df.rename(columns={'depth': 'depth_percentage'})


# changing column names
df = df.rename(columns={"'x'": 'length'})
df = df.rename(columns={"'y'": 'width'})
df = df.rename(columns={"'z'": 'depth'})


print(df.dtypes)

carat               float64
cut                  object
color                object
clarity              object
depth_percentage    float64
table               float64
price               float64
length              float64
width               float64
depth               float64
dtype: object


# check for missing value
df.isnull().sum()

carat               0
cut                 0
color               0
clarity             0
depth_percentage    0
table               0
price               0
length              0
width               0
depth               0
dtype: int64


# check for 0s in each column
df.isin([0]).sum()

carat                0
cut                  0
color                0
clarity              0
depth_percentage     0
table                0
price                0
length               8
width                7
depth               20
dtype: int64


# create new variable: price_range
df['price_range'] = pd.cut(df['price'],
                                 [300,900,2000,4000,6500,1e6],
                                 labels=['0','1','2','3','4'])
df.price_range.describe()

# convert price_range to numeric
df["price_range"]=df["price_range"].astype(str).astype(int)

# drop the price column
df.drop(['price'], axis=1, inplace=True)


# now lets group with the new variable
df_grouped = df.groupby(by=['cut','price_range'])
print ("number of cut falling in specific price range:")
df_grouped.count()

number of cut falling in specific price range:


df["price_range"]=df["price_range"].astype(str).astype(int)


# print row index of rows with zeros in length column
df.loc[df['length'] == 0]


# print row index of rows with zeros in width column
df.loc[df['width'] == 0]


# print row index of rows with zeros in depth column
df.loc[df['depth'] == 0]


# print out total number of rows with zeros in length, width, and depth column
print(f"Total number of rows with zeros in length, width, and depth column: {df[df['length'] == 0].shape[0] + df[df['width'] == 0].shape[0] + df[df['depth'] == 0].shape[0]}")

# print out percentage of rows with zeros in length, width, and depth column
print(f"Percentage of rows with zeros in length, width, and depth column: {(df[df['length'] == 0].shape[0] + df[df['width'] == 0].shape[0] + df[df['depth'] == 0].shape[0]) / df.shape[0] * 100}%")

Total number of rows with zeros in length, width, and depth column: 35
Percentage of rows with zeros in length, width, and depth column: 0.06488691138301816%


df.loc[df['length'] == 0, 'length'] = df['length'].median()


df.loc[df['depth'] == 0, 'depth'] = df['depth'].median()


df.loc[df['width'] == 0, 'width'] = df['width'].median()


# drop duplicates
df.drop_duplicates(keep = False, inplace=True)

print(f"Number of observations after deleting duplicates: {df.shape[0]}")

Number of observations after deleting duplicates: 53299


# dataset statistics
df.describe()


%matplotlib inline
from matplotlib import pyplot as plt
import seaborn as sns

sns.set()
sns.pairplot(df, hue = "carat", height = 2)

<seaborn.axisgrid.PairGrid at 0x26fbf6aadc0>


import seaborn as sns

cmap = sns.set(style="darkgrid")
f, ax = plt.subplots(figsize=(8, 8))
sns.heatmap(df.corr(), cmap=cmap, annot=True)

C:\Users\Ibrahim\AppData\Local\Temp\ipykernel_5788\3684316204.py:5: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.
  sns.heatmap(df.corr(), cmap=cmap, annot=True)

<Axes: >


# print out unique color values
print(df['color'].unique())

["b'E'" "b'I'" "b'J'" "b'H'" "b'F'" "b'G'" "b'D'"]


# one hot encode the color feature
if "color" in df.columns:
    df = pd.get_dummies(df, columns=["color"])

# rename the color columns
df.rename(columns = {
    "color_b\'D\'": "color_D",
    "color_b\'E\'": "color_E",
    "color_b\'F\'": "color_F",
    "color_b\'G\'": "color_G",
    "color_b\'H\'": "color_H",
    "color_b\'I\'": "color_I",
    "color_b\'J\'": "color_J"
}, inplace = True)


# print out unique clarity values
print(df['clarity'].unique())

["b'SI2'" "b'SI1'" "b'VS1'" "b'VS2'" "b'VVS2'" "b'VVS1'" "b'I1'" "b'IF'"]


# one hot encode the clarity feature
if "clarity" in df.columns:
    df = pd.get_dummies(df, columns=["clarity"])

# rename the clarity columns
df.rename(columns = {
    "clarity_b\'IF\'": "clarity_IF",
    "clarity_b\'VVS1\'": "clarity_VVS1",
    "clarity_b\'VVS2\'": "clarity_VVS2",
    "clarity_b\'VS1\'": "clarity_VS1",
    "clarity_b\'VS2\'": "clarity_VS2",
    "clarity_b\'SI1\'": "clarity_SI1",
    "clarity_b\'SI2\'": "clarity_SI2",
    "clarity_b\'I1\'": "clarity_I1"
}, inplace = True)


# print out unique cut values
print(df['cut'].unique())

["b'Ideal'" "b'Premium'" "b'Good'" "b'Very Good'" "b'Fair'"]


# for each value in cut column, clean the string
if "cut" in df.columns:
    df = pd.get_dummies(df, columns=["cut"])

# rename the cut columns
df.rename(columns = {
    "cut_b\'Fair\'": "cut_Fair",
    "cut_b\'Good\'": "cut_Good",
    "cut_b\'Very Good\'": "cut_Very Good",
    "cut_b\'Premium\'": "cut_Premium",
    "cut_b\'Ideal\'": "cut_Ideal"
}, inplace = True)


# print out columns
print(df.dtypes)

carat               float64
depth_percentage    float64
table               float64
length              float64
width               float64
depth               float64
price_range           int32
color_D               uint8
color_E               uint8
color_F               uint8
color_G               uint8
color_H               uint8
color_I               uint8
color_J               uint8
clarity_I1            uint8
clarity_IF            uint8
clarity_SI1           uint8
clarity_SI2           uint8
clarity_VS1           uint8
clarity_VS2           uint8
clarity_VVS1          uint8
clarity_VVS2          uint8
cut_Fair              uint8
cut_Good              uint8
cut_Ideal             uint8
cut_Premium           uint8
cut_Very Good         uint8
dtype: object


from sklearn.model_selection import ShuffleSplit

df2 = copy.deepcopy(df)

# we want to predict the X and y data as follows:
if 'price_range' in df:
    # get target label
    y = df['price_range'].values 

    # delete the price_range column from data
    del df['price_range']

    # standardize the data    
    norm_features = ['length', 'width', 'depth', 'carat', 'depth_percentage','table']

    df[norm_features] = (df[norm_features]-df[norm_features].mean()) / df[norm_features].std()
    
    # convert to numpy array
    X = df.to_numpy()

# cross validation for model
num_cv_iterations = 3
num_instances = len(y)
cv_object = ShuffleSplit(n_splits = num_cv_iterations, test_size  = 0.2)
                         
print(cv_object)

ShuffleSplit(n_splits=3, random_state=None, test_size=0.2, train_size=None)


plotly.offline.init_notebook_mode()

# create graph
graph1 = {'x': np.unique(y), 'y': np.bincount(y)/len(y), 'type': 'bar'}

fig = dict()
fig['data'] = [graph1]
fig['layout'] = {'title': 'Multi Class Distribution',
                'autosize':False,
                'width':400,
                'height':400}

# plot graph
plotly.offline.iplot(fig)


from sklearn.preprocessing import StandardScaler
from sklearn import metrics as mt

X = StandardScaler().fit(X).transform(X)


from scipy.special import expit

# create base binary logistic regression class
class BinaryLogisticRegression:
    def __init__(self, eta, iterations = 20, C = 0.001):
        self.eta = eta
        self.iters = iterations
        self.C = C
        
    def __str__(self):
        if(hasattr(self,'w_')):
            return 'Binary Logistic Regression Object with coefficients:\n'+ str(self.w_)
        else:
            return 'Untrained Binary Logistic Regression Object'
        
    @staticmethod
    def _add_bias(X):
        return np.hstack((np.ones((X.shape[0],1)),X)) 
    
    @staticmethod
    def _sigmoid(theta):
        return expit(theta)
    
    # vectorized gradient calculation with regularization using L2 Norm
    def _get_gradient(self,X,y):
        ydiff = y-self.predict_proba(X,add_bias=False).ravel() 
        gradient = np.mean(X * ydiff[:,np.newaxis], axis=0) 
        
        gradient = gradient.reshape(self.w_.shape)
        gradient[1:] += -2 * self.w_[1:] * self.C
        
        return gradient
    
    def predict_proba(self,X,add_bias=True):
        # add bias term if requested
        Xb = self._add_bias(X) if add_bias else X
        return self._sigmoid(Xb @ self.w_)
    
    def predict(self,X):
        return (self.predict_proba(X)>0.5) 
    
    
    def fit(self, X, y):
         # add bias term
        Xb = self._add_bias(X)
        num_samples, num_features = Xb.shape
        
        self.w_ = np.zeros((num_features,1))

        # for as many as the max iterations
        for _ in range(self.iters):
            gradient = self._get_gradient(Xb,y)

             # multiply by learning rate 
            self.w_ += gradient*self.eta


class VectorBinaryLogisticRegression(BinaryLogisticRegression):
    @staticmethod
    def _sigmoid(theta):
        return expit(theta)
    
    def _get_gradient(self,X,y):
        # get y difference
        ydiff = y-self.predict_proba(X,add_bias=False).ravel()

         # make ydiff a column vector and multiply through
        gradient = np.mean(X * ydiff[:,np.newaxis], axis=0)
        
        return gradient.reshape(self.w_.shape)


from scipy.optimize import fmin_bfgs
from numpy import ma

class BFGSBinaryLogisticRegression(BinaryLogisticRegression):
    @staticmethod
    def objective_function(w,X,y,C):
        g = expit(X @ w)
        
        return -np.sum(ma.log(g[y==1]))-np.sum(ma.log(1-g[y==0])) + C*sum(w**2) 

    @staticmethod
    def objective_gradient(w,X,y,C):
        g = expit(X @ w)

         # get y difference
        ydiff = y-g
        gradient = np.mean(X * ydiff[:,np.newaxis], axis=0)
        gradient = gradient.reshape(w.shape)
        gradient[1:] += -2 * w[1:] * C
        return -gradient
    
    def fit(self, X, y):
         # add bias term
        Xb = self._add_bias(X)
        num_samples, num_features = Xb.shape
        
        self.w_ = fmin_bfgs(self.objective_function, # what to optimize
                            np.zeros((num_features,1)), # starting point
                            fprime=self.objective_gradient, # gradient function
                            args=(Xb,y,self.C), # extra args for gradient and objective function
                            gtol=1e-03, # stopping criteria for gradient, |v_k|
                            maxiter=self.iters, # stopping criteria iterations
                            disp=False)
        
        self.w_ = self.w_.reshape((num_features,1))


class MultiClassLogisticRegression:
    def __init__(self, eta, iterations=20, 
                 C=0.0001, 
                 solver=BFGSBinaryLogisticRegression):
        self.eta = eta
        self.iters = iterations
        self.C = C
        self.solver = solver
        self.classifiers_ = []
    
    def __str__(self):
        if(hasattr(self,'w_')):
            return 'MultiClass Logistic Regression Object with coefficients:\n'+ str(self.w_)
        else:
            return 'Untrained MultiClass Logistic Regression Object'
        
    def fit(self,X,y):
        num_samples, num_features = X.shape

         # get each unique class value
        self.unique_ = np.sort(np.unique(y))
        num_unique_classes = len(self.unique_)
        self.classifiers_ = []

         # for each unique value
        for i,yval in enumerate(self.unique_):
            y_binary = np.array(y==yval).astype(int)
            
            hblr = self.solver(eta=self.eta,iterations=self.iters,C=self.C)
            hblr.fit(X,y_binary)

            # add the trained classifier to the list
            self.classifiers_.append(hblr)
            
        self.w_ = np.hstack([x.w_ for x in self.classifiers_]).T
        
    def predict_proba(self,X):
        probs = []
        for hblr in self.classifiers_:
            # get probability for each classifier
            probs.append(hblr.predict_proba(X).reshape((len(X),1)))
        
        # make into single matrix
        return np.hstack(probs)
    
    def predict(self,X):
        return np.argmax(self.predict_proba(X),axis=1)


class LogisticRegression:
    def __init__(self, eta, iterations=20):
        self.eta = eta
        self.iters = iterations
       
    def __str__(self):
        if(hasattr(self,'w_')):
            return 'MultiClass Logistic Regression Object with coefficients:\n'+ str(self.w_)
        else:
            return 'Untrained MultiClass Logistic Regression Object'
        
    def fit(self,X,y):
        num_samples, num_features = X.shape

         # get each unique class value
        self.unique_ = np.unique(y)
        num_unique_classes = len(self.unique_)
        self.classifiers_ = [] 

        # for each unique value
        for i,yval in enumerate(self.unique_):

            # create a binary problem
            y_binary = (y==yval)

            # train the binary classifier for this class
            blr = VectorBinaryLogisticRegression(self.eta,
                                                 self.iters)
            blr.fit(X,y_binary)

            # add the trained classifier to the list
            self.classifiers_.append(blr)
            
        # save all the weights into one matrix, separate column for each class
        self.w_ = np.hstack([x.w_ for x in self.classifiers_]).T
        
    def predict_proba(self,X):
        probs = []
        for blr in self.classifiers_:
            # get probability for each classifier
            probs.append(blr.predict_proba(X))

         # make into single matrix
        return np.hstack(probs)
    
    def predict(self,X):
        return self.unique_[np.argmax(self.predict_proba(X),axis=1)]


class StochasticLogisticRegression(BinaryLogisticRegression):
    # stochastic gradient calculation 
    def _get_gradient(self,X,y):
        # grab random instance
        idx = int(np.random.rand()*len(y)) 

        # get scalar y difference
        ydiff = y[idx]-self.predict_proba(X[idx],add_bias=False)

        # make ydiff a column vector and multiply through
        gradient = X[idx] * ydiff[:,np.newaxis]
        
        gradient = gradient.reshape(self.w_.shape)
        gradient[1:] += -2 * self.w_[1:] * self.C
        
        return gradient


from numpy.linalg import pinv
class HessianBinaryLogisticRegression(BinaryLogisticRegression):
    # just overwrite gradient function
    def _get_gradient(self,X,y):
         # get sigmoid value for all classes
        g = self.predict_proba(X,add_bias=False).ravel()

        # calculate the hessian
        hessian = X.T @ np.diag(g*(1-g)) @ X - 2 * self.C

        # get y difference
        ydiff = y-g

        # make ydiff a column vector and multiply through
        gradient = np.sum(X * ydiff[:,np.newaxis], axis=0) 
        gradient = gradient.reshape(self.w_.shape)
        gradient[1:] += -2 * self.w_[1:] * self.C
        
        return pinv(hessian) @ gradient


class RegularizedBinaryLogisticRegression(VectorBinaryLogisticRegression):
    # extend init functions
    def __init__(self, C=0.0, **kwds):        
        # need to add to the original initializer 
        self.C = C
        # but keep other keywords
        super().__init__(**kwds) # call parent initializer
        
        
    # extend previous class to change functionality
    def _get_gradient(self,X,y):
        # call get gradient from previous class
        gradient = super()._get_gradient(X,y)
        
        # add in regularization (to all except bias term)
        gradient[1:] += -2 * self.w_[1:] * self.C
        return gradient


class RegularizedLogisticRegression(LogisticRegression):
    def __init__(self, C=0.0, **kwds):        
        # need to add to the original initializer 
        self.C = C

        super().__init__(**kwds)
        
    def fit(self,X,y):
        num_samples, num_features = X.shape

         # get each unique class value
        self.unique_ = np.unique(y)
        num_unique_classes = len(self.unique_)
        self.classifiers_ = []


        # for each unique value
        for i,yval in enumerate(self.unique_): 
            # create a binary problem
            y_binary = y==yval 

            blr = RegularizedBinaryLogisticRegression(eta=self.eta,
                                                      iterations=self.iters,
                                                      C=self.C)
            blr.fit(X,y_binary)
            
            # add the trained classifier to the list
            self.classifiers_.append(blr)
            
        # save all the weights into one matrix, separate column for each class
        self.w_ = np.hstack([x.w_ for x in self.classifiers_]).T


%%time
# Steepest Descent
lr = LogisticRegression(0.1,1500)
lr.fit(X,y)

yhat1 = lr.predict(X)
print(f'Accuracy of Model with Steepest Ascent: {mt.accuracy_score(y,yhat1) * 100}%')

# plot heatmap of confusion matrix
class_names = df2.price_range.unique()
fig,ax  = plt.subplots(figsize=(10,10))

tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names)
plt.yticks(tick_marks, class_names)

# create heatmap
sns.heatmap(pd.DataFrame(mt.confusion_matrix(y,yhat1)), annot = True, cmap = "YlGnBu" ,fmt = 'g')

plt.tight_layout()
plt.title('Confusion matrix on Correct and Incorrect Predictions of the Steepest Descent Model', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

plt.show()

Accuracy of Model with Steepest Ascent: 75.27345728812924%

CPU times: total: 57.2 s
Wall time: 49.2 s


%%time
# Binary Stochastic Gradient Descent
slr = MultiClassLogisticRegression(eta=0.05, iterations=2500, C=0.0001, solver = StochasticLogisticRegression)
slr.fit(X,y)

yhat2 = slr.predict(X)
print(f'Accuracy of Model with Stochastic Gradient Descent: {mt.accuracy_score(y,yhat2) * 100}%')

# plot heatmap of confusion matrix
class_names = df2.price_range.unique()
fig,ax  = plt.subplots(figsize=(10,10))

tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names)
plt.yticks(tick_marks, class_names)

# create heatmap
sns.heatmap(pd.DataFrame(mt.confusion_matrix(y,yhat2)), annot = True, cmap = "YlGnBu" ,fmt = 'g')

plt.tight_layout()
plt.title('Confusion matrix on Correct and Incorrect Predictions of the Stochastic Gradient Descent Model', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

plt.show()

Accuracy of Model with Stochastic Gradient Descent: 67.00500947484943%

CPU times: total: 438 ms
Wall time: 431 ms


%%time
# Newton's Method
hlr = HessianBinaryLogisticRegression(eta=0.1, iterations=4, C=0.01) 
hlr.fit(X,y)

yhat3 = hlr.predict(X)
print(f'Accuracy of Model with Newton\'s Method: {mt.accuracy_score(y,yhat3) * 100}%')

# plot heatmap of confusion matrix
class_names = df2.price_range.unique()
fig,ax  = plt.subplots(figsize=(10,10))

tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names)
plt.yticks(tick_marks, class_names)

# create heatmap
sns.heatmap(pd.DataFrame(mt.confusion_matrix(y,yhat3)), annot = True, cmap = "YlGnBu" ,fmt = 'g')

plt.tight_layout()
plt.title('Confusion matrix on Correct and Incorrect Predictions of Newton\'s Method Model', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

plt.show()

Accuracy of Model with Newton's Method: 17.390570179553087%

CPU times: total: 7min 7s
Wall time: 26.7 s


# import the necessary libraries
from sklearn.linear_model import LogisticRegression
from sklearn import metrics

logModel = LogisticRegression(random_state=16)

# fit the model
logModel.fit(X, y)

# make predictions
yhat4 = logModel.predict(X)

# print accuracy percentage of model
print("Accuracy: ", metrics.accuracy_score(y, yhat4) * 100,"%")

Accuracy:  89.84221092328187 %

c:\users\ibrahim\code\saminas_homework\machine_learning_imag_processing\.venv\image_processing\lib\site-packages\sklearn\linear_model\_logistic.py:458: ConvergenceWarning:

lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression


# import the necessary libraries
from sklearn import metrics
import seaborn as sns
import matplotlib.pyplot as plt

# print the confusion matrix
c_matrix= metrics.confusion_matrix(y, yhat4)

class_names = df2.price_range.unique()
fig,ax  = plt.subplots(figsize=(10,10))

tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names)
plt.yticks(tick_marks, class_names)

# create heatmap
sns.heatmap(pd.DataFrame(c_matrix), annot = True, cmap = "YlGnBu" ,fmt = 'g')

plt.tight_layout()
plt.title('Confusion matrix on Correct and Incorrect Predictions of the Scikit Model', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

plt.show()


# print accuracy of steepest ascent model
print("Accuracy: ", metrics.accuracy_score(y, yhat1) * 100,"%")

# print accuracy of scikit model
print("Accuracy: ", metrics.accuracy_score(y, yhat4) * 100,"%")

Accuracy:  75.27345728812924 %
Accuracy:  89.84221092328187 %


# print the confusion matrix
class_names = df2.price_range.unique()
fig,ax  = plt.subplots(figsize=(20,10))

tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names)
plt.yticks(tick_marks, class_names)

plt.subplot(1,2,1)
# create heatmap
sns.heatmap(pd.DataFrame(mt.confusion_matrix(y,yhat1)), annot = True, cmap = "YlGnBu" ,fmt = 'g')

plt.tight_layout()
plt.title('Confusion matrix on Correct and Incorrect Predictions of the Steepest Descent Model', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

plt.subplot(1,2,2)
# create heatmap
sns.heatmap(pd.DataFrame(mt.confusion_matrix(y,yhat4)), annot = True, cmap = "YlGnBu" ,fmt = 'g')

plt.tight_layout()
plt.title('Confusion matrix on Correct and Incorrect Predictions of the Scikit Model', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')

plt.show()

C:\Users\Ibrahim\AppData\Local\Temp\ipykernel_5788\3598540272.py:9: MatplotlibDeprecationWarning:

Auto-removal of overlapping axes is deprecated since 3.6 and will be removed two minor releases later; explicitly call ax.remove() as needed.


params = dict(eta=0.01, iterations=500)


class MSEVectorBinaryLogisticRegression(BinaryLogisticRegression):
    # inherit from our previous class to get same functionality
    @staticmethod
    def _sigmoid(theta):
        # increase stability, redefine sigmoid operation
        return expit(theta) #1/(1+np.exp(-theta))
    # but overwrite the gradient calculation
    def _get_gradient(self,X,y):
        ydiff = np.square(y-self.predict_proba(X,add_bias=False).ravel()) # get y difference
        gradient = np.mean(X * ydiff[:,np.newaxis], axis=0) # make ydiff a column vector and multiply through
        return gradient.reshape(self.w_.shape)
    
# use same params as defined above
MSEblr = MSEVectorBinaryLogisticRegression(**params)
MSEblr.fit(X,y)


# allow for the user to specify the algorithm they want to sovle the binary case
class MseLogisticRegression:
    def __init__(self, eta, iterations=20, 
                 C=0.0001, 
                 solver=MSEVectorBinaryLogisticRegression):
        self.eta = eta
        self.iters = iterations
        self.C = C
        self.solver = solver
        self.classifiers_ = []
        # internally we will store the weights as self.w_ to keep with sklearn conventions
    
    def __str__(self):
        if(hasattr(self,'w_')):
            return 'MultiClass Logistic Regression Object with coefficients:\n'+ str(self.w_) # is we have trained the object
        else:
            return 'Untrained MultiClass Logistic Regression Object'
        
    def fit(self,X,y):
        num_samples, num_features = X.shape
        self.unique_ = np.sort(np.unique(y)) # get each unique class value
        num_unique_classes = len(self.unique_)
        self.classifiers_ = []
        for i,yval in enumerate(self.unique_): # for each unique value
            y_binary = np.array(y==yval).astype(int) # create a binary problem
            # train the binary classifier for this class
            
            hblr = self.solver(eta=self.eta,iterations=self.iters,C=self.C)
            hblr.fit(X,y_binary)

            # add the trained classifier to the list
            self.classifiers_.append(hblr)
            
        # save all the weights into one matrix, separate column for each class
        self.w_ = np.hstack([x.w_ for x in self.classifiers_]).T
        
    def predict_proba(self,X):
        probs = []
        for hblr in self.classifiers_:
            probs.append(hblr.predict_proba(X).reshape((len(X),1))) # get probability for each classifier
        
        return np.hstack(probs) # make into single matrix
    
    def predict(self,X):
        return np.argmax(self.predict_proba(X),axis=1) # take argmax along row


%%time
lr = MseLogisticRegression(eta=1, iterations=500, C=0.001, solver=BFGSBinaryLogisticRegression)
lr.fit(X,y)

yhat = lr.predict(X)
print('Accuracy of: ',mt.accuracy_score(y,yhat))

Accuracy of:  0.7801459689675229
CPU times: total: 2.77 s
Wall time: 2.44 s


bfgslr = BFGSBinaryLogisticRegression(_,iterations = 2, C = 0.001) # note that we need only a few iterations here
bfgslr.fit(X,y)

yhat4 = bfgslr.predict(X)
print('Accuracy of: ',mt.accuracy_score(y,yhat4))

Accuracy of:  0.24914163492748456


# allow for the user to specify the algorithm they want to sovle the binary case
class MultiClassLogisticRegression:
    def __init__(self, eta, iterations=20, 
                 C=0.0001, 
                 solver=BFGSBinaryLogisticRegression):
        self.eta = eta
        self.iters = iterations
        self.C = C
        self.solver = solver
        self.classifiers_ = []
        # internally we will store the weights as self.w_ to keep with sklearn conventions
    
    def __str__(self):
        if(hasattr(self,'w_')):
            return 'MultiClass Logistic Regression Object with coefficients:\n'+ str(self.w_) # is we have trained the object
        else:
            return 'Untrained MultiClass Logistic Regression Object'
        
    def fit(self,X,y):
        num_samples, num_features = X.shape
        self.unique_ = np.sort(np.unique(y)) # get each unique class value
        num_unique_classes = len(self.unique_)
        self.classifiers_ = []
        for i,yval in enumerate(self.unique_): # for each unique value
            y_binary = np.array(y==yval).astype(int) # create a binary problem
            # train the binary classifier for this class
            
            hblr = self.solver(eta=self.eta,iterations=self.iters,C=self.C)
            hblr.fit(X,y_binary)

            # add the trained classifier to the list
            self.classifiers_.append(hblr)
            
        # save all the weights into one matrix, separate column for each class
        self.w_ = np.hstack([x.w_ for x in self.classifiers_]).T
        
    def predict_proba(self,X):
        probs = []
        for hblr in self.classifiers_:
            probs.append(hblr.predict_proba(X).reshape((len(X),1))) # get probability for each classifier
        
        return np.hstack(probs) # make into single matrix
    
    def predict(self,X):
        return np.argmax(self.predict_proba(X),axis=1) # take argmax along row


# run logistic regression and vary some parameters
from sklearn import metrics as mt

# first we create a reusable logisitic regression object
#   here we can setup the object with different learning parameters and constants
lr_clf = MultiClassLogisticRegression(eta=1,
                                  iterations=500,
                                  C=0.0001,
                                  solver=BFGSBinaryLogisticRegression
                                 )
# now we can use the cv_object that we setup before to iterate through the 
#    different training and testing sets. Each time we will reuse the logisitic regression 
#    object, but it gets trained on different data each time we use it.

iter_num=0
# the indices are the rows used for training and testing in each iteration
for train_indices, test_indices in cv_object.split(X,y): 
    # I will create new variables here so that it is more obvious what 
    # the code is doing (you can compact this syntax and avoid duplicating memory,
    # but it makes this code less readable)
    X_train = X[train_indices]
    y_train = y[train_indices]
    
    X_test = X[test_indices]
    y_test = y[test_indices]
    
    # train the reusable logisitc regression model on the training data
    lr_clf.fit(X_train,y_train)  # train object
    y_hat = lr_clf.predict(X_test) # get test set precitions

    # now let's get the accuracy and confusion matrix for this iterations of training/testing
    acc = mt.accuracy_score(y_test,y_hat)
    conf = mt.confusion_matrix(y_test,y_hat)
    print("====Iteration",iter_num," ====")
    print("accuracy", acc )
    print("confusion matrix\n",conf)
    iter_num+=1
    
# Also note that every time you run the above code
#   it randomly creates a new training and testing set, 
#   so accuracy will be different each time

====Iteration 0  ====
accuracy 0.8273921200750469
confusion matrix
 [[2289  160    1    0    0]
 [ 319 1781  249    0    0]
 [   0  307 1436  300    0]
 [   1    6  236 1365  193]
 [   0    0    1   67 1949]]
====Iteration 1  ====
accuracy 0.8227954971857411
confusion matrix
 [[2285  158    1    0    0]
 [ 355 1738  270    0    0]
 [   0  308 1405  308    0]
 [   0    7  208 1401  200]
 [   0    0    4   70 1942]]
====Iteration 2  ====
accuracy 0.8314258911819887
confusion matrix
 [[2336  156    0    0    0]
 [ 305 1724  262    0    0]
 [   2  316 1447  283    0]
 [   0    7  216 1365  186]
 [   0    0    4   60 1991]]


from ipywidgets import widgets as wd

num_cv_iterations = 10
num_instances = len(y)
cv_object = ShuffleSplit(n_splits=num_cv_iterations,
                         test_size  = 0.2)

def lr_explor(cost):
    print('Running')
    lr_clf = MultiClassLogisticRegression(eta=1,
                                  iterations=500,
                                  C=float(cost),
                                  solver=BFGSBinaryLogisticRegression
                                 )
    acc = []
    for iter_num, (train_indices, test_indices) in enumerate(cv_object.split(X,y)):
        lr_clf.fit(X[train_indices],y[train_indices])  # train object
        y_hat = lr_clf.predict(X[test_indices]) # get test set predictions
        acc.append(mt.accuracy_score(y[test_indices],y_hat))
        
    acc = np.array(acc)
    print(acc.mean(),'+-',2.7*acc.std())
        
wd.interact(lr_explor,cost=list(np.logspace(-4,1,15)),__manual=True)

interactive(children=(Dropdown(description='cost', options=(0.0001, 0.00022758459260747887, 0.0005179474679231…

<function __main__.lr_explor(cost)>


%%time

# alternatively, we can also graph out the values using boxplots
num_cv_iterations = 10
num_instances = len(y)
cv_object = ShuffleSplit(n_splits=num_cv_iterations,
                         test_size  = 0.2)

def lr_explor(cost):
    lr_clf = MultiClassLogisticRegression(eta=1,
                                  iterations=500,
                                  C=float(cost),
                                  solver=BFGSBinaryLogisticRegression
                                 )
    acc = []
    for iter_num, (train_indices, test_indices) in enumerate(cv_object.split(X,y)):
        lr_clf.fit(X[train_indices],y[train_indices])  # train object
        y_hat = lr_clf.predict(X[test_indices]) # get test set predictions
        acc.append(mt.accuracy_score(y[test_indices],y_hat))
        
    acc = np.array(acc)
    return acc

costs = np.logspace(-5,1,20)
accs = []
for c in costs:
    accs.append(lr_explor(c))

CPU times: total: 6min 37s
Wall time: 5min 31s


# now show a boxplot of the data across c
from matplotlib import pyplot as plt
%matplotlib inline

plt.boxplot(accs)
plt.xticks(range(1,len(costs)+1),['%.4f'%(c) for c in costs],rotation='vertical')
plt.xlabel('C')
plt.ylabel('validation accuracy')
plt.show()


%%time
lr = MultiClassLogisticRegression(eta=1,
                                  iterations=500,
                                  C=0.001,
                                  solver=BFGSBinaryLogisticRegression
                                 )
lr.fit(X,y)
print(lr)

yhat = lr.predict(X)
print('Accuracy of: ',accuracy_score(y,yhat))

MultiClass Logistic Regression Object with coefficients:
[[-7.43513578e+00 -1.30449904e+00  1.82580447e-02 -1.81746589e-01
  -2.64026020e+00 -2.50768781e+00 -2.55068645e+00 -4.73933978e-01
  -3.79413903e-01 -2.16055418e-01 -2.74123176e-02  3.60969134e-01
   5.66155500e-01  4.38001620e-01  2.75640245e-01 -5.36650104e-01
   4.08579709e-01  5.73903426e-01 -1.96701761e-01  9.22558919e-04
  -5.60397354e-01 -3.95241951e-01 -1.50433423e-01  1.48452725e-01
  -9.61263084e-02 -1.57340966e-01  2.36215517e-01]
 [-1.99941906e+00 -4.07153588e+00 -3.15012236e-02  7.18268896e-03
   9.20573324e-01  4.40829759e-01  9.14292689e-01  9.27647179e-02
   8.55000267e-02  1.89618755e-02  2.68520743e-02 -1.28301612e-01
  -1.51520210e-01  4.40133747e-02  1.02730835e-01  1.10235723e-01
  -4.79696972e-02 -1.10337441e-01  9.13837653e-03  3.11201732e-02
   8.96042602e-02 -2.95465082e-02  9.60909389e-02 -4.31997847e-02
   5.19241627e-02  4.90909926e-02 -1.21510945e-01]
 [-1.72683155e+00 -4.99047880e+00  1.14335731e-01  1.69083301e-01
   1.70209867e+00  1.55883990e+00  1.55502148e+00 -2.40995100e-02
   6.37551723e-03  4.98727510e-02 -6.78023038e-02 -4.22641866e-02
   7.55839034e-02  2.98906533e-02  1.43113579e-01 -5.23278017e-02
  -2.64694528e-02  1.59819927e-01 -2.83810422e-02 -1.03747999e-01
  -4.43115014e-02  3.03153924e-02  8.00034953e-02  3.16831773e-02
   6.74306579e-03 -8.25780755e-02  2.39927976e-02]
 [-2.46600858e+00 -3.70381977e+00  1.50585276e-01  1.04001245e-01
   1.80472273e+00  1.52193469e+00  1.63054987e+00  3.38299308e-02
  -2.41419179e-02 -2.96648767e-02 -3.72413018e-02  1.03436161e-01
  -2.86532499e-02 -1.83425631e-02 -4.16141092e-02 -4.78540331e-01
   3.50933205e-01  2.22325428e-01 -5.00396510e-02  2.77472900e-02
  -2.13657179e-01 -2.85254650e-01 -1.92607839e-02  7.21310770e-02
  -9.59292798e-02  1.71178337e-02  5.30153626e-02]
 [-6.25907170e+00  2.48069909e+00 -2.49208932e-02 -7.21848750e-02
   1.73291457e+00  1.69324600e+00  1.64840286e+00  3.30656499e-01
   3.34569222e-01  3.00556510e-01  1.73881675e-01 -3.02006099e-01
  -5.42128242e-01 -6.88151437e-01 -8.08599163e-01  5.72484245e-01
  -6.71072951e-01 -1.16572076e+00  5.82361574e-01  2.31802741e-01
   6.42714046e-01  8.52669161e-01 -2.45161702e-01 -1.12645611e-01
   1.63526432e-01 -5.31788752e-02  4.05805388e-02]]

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
File <timed exec>:10

NameError: name 'accuracy_score' is not defined


from matplotlib import pyplot as plt
import copy
%matplotlib inline
plt.style.use('ggplot')

def plot_decision_boundaries(lr,Xin,y,title=''):
    Xb = copy.deepcopy(Xin)
    lr.fit(Xb[:,:2],y) # train only on two features

    h=0.01
    # create a mesh to plot in
    x_min, x_max = Xb[:, 0].min() - 1, Xb[:, 0].max() + 1
    y_min, y_max = Xb[:, 1].min() - 1, Xb[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    # get prediction values
    Z = lr.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.5)

    # Plot also the training points
    plt.scatter(Xb[:, 0], Xb[:, 1], c=y, cmap=plt.cm.Paired)
    plt.xlabel('Independent Value')
    plt.ylabel('Dpendent value')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title(title)
    plt.show()
    
lr = BFGSBinaryLogisticRegression(0.1,1500) # this is still OUR LR implementation, not sklearn
plot_decision_boundaries(lr,X,y)


params = dict(eta=0.01, iterations=500)


class MSEVectorBinaryLogisticRegression(BinaryLogisticRegression):
    # inherit from our previous class to get same functionality
    @staticmethod
    def _sigmoid(theta):
        # increase stability, redefine sigmoid operation
        return expit(theta) #1/(1+np.exp(-theta))
    # but overwrite the gradient calculation
    def _get_gradient(self,X,y):
        ydiff = np.square(y-self.predict_proba(X,add_bias=False).ravel()) # get y difference
        gradient = np.mean(X * ydiff[:,np.newaxis], axis=0) # make ydiff a column vector and multiply through
        return gradient.reshape(self.w_.shape)
    
# use same params as defined above
MSEblr = MSEVectorBinaryLogisticRegression(**params)
MSEblr.fit(X,y)


# allow for the user to specify the algorithm they want to sovle the binary case
class MseLogisticRegression:
    def __init__(self, eta, iterations=20, 
                 C=0.0001, 
                 solver=MSEVectorBinaryLogisticRegression):
        self.eta = eta
        self.iters = iterations
        self.C = C
        self.solver = solver
        self.classifiers_ = []
        # internally we will store the weights as self.w_ to keep with sklearn conventions
    
    def __str__(self):
        if(hasattr(self,'w_')):
            return 'MultiClass Logistic Regression Object with coefficients:\n'+ str(self.w_) # is we have trained the object
        else:
            return 'Untrained MultiClass Logistic Regression Object'
        
    def fit(self,X,y):
        num_samples, num_features = X.shape
        self.unique_ = np.sort(np.unique(y)) # get each unique class value
        num_unique_classes = len(self.unique_)
        self.classifiers_ = []
        for i,yval in enumerate(self.unique_): # for each unique value
            y_binary = np.array(y==yval).astype(int) # create a binary problem
            # train the binary classifier for this class
            
            hblr = self.solver(eta=self.eta,iterations=self.iters,C=self.C)
            hblr.fit(X,y_binary)

            # add the trained classifier to the list
            self.classifiers_.append(hblr)
            
        # save all the weights into one matrix, separate column for each class
        self.w_ = np.hstack([x.w_ for x in self.classifiers_]).T
        
    def predict_proba(self,X):
        probs = []
        for hblr in self.classifiers_:
            probs.append(hblr.predict_proba(X).reshape((len(X),1))) # get probability for each classifier
        
        return np.hstack(probs) # make into single matrix
    
    def predict(self,X):
        return np.argmax(self.predict_proba(X),axis=1) # take argmax along row


%%time
lr = MseLogisticRegression(eta=1, iterations=500, C=0.001, solver=BFGSBinaryLogisticRegression)
lr.fit(X,y)

yhat = lr.predict(X)
print('Accuracy of: ',mt.accuracy_score(y,yhat))

Accuracy of:  0.7801459689675229
CPU times: total: 2.69 s
Wall time: 2.4 s

		carat	color	clarity	depth_percentage	table	length	width	depth
cut	price_range
b'Fair'	0	76	76	76	76	76	76	76	76
	1	312	312	312	312	312	312	312	312
	2	591	591	591	591	591	591	591	591
	3	344	344	344	344	344	344	344	344
	4	287	287	287	287	287	287	287	287
b'Good'	0	1011	1011	1011	1011	1011	1011	1011	1011
	1	825	825	825	825	825	825	825	825
	2	1196	1196	1196	1196	1196	1196	1196	1196
	3	1063	1063	1063	1063	1063	1063	1063	1063
	4	811	811	811	811	811	811	811	811
b'Ideal'	0	5690	5690	5690	5690	5690	5690	5690	5690
	1	5912	5912	5912	5912	5912	5912	5912	5912
	2	3755	3755	3755	3755	3755	3755	3755	3755
	3	2577	2577	2577	2577	2577	2577	2577	2577
	4	3617	3617	3617	3617	3617	3617	3617	3617
b'Premium'	0	2631	2631	2631	2631	2631	2631	2631	2631
	1	2767	2767	2767	2767	2767	2767	2767	2767
	2	2281	2281	2281	2281	2281	2281	2281	2281
	3	2809	2809	2809	2809	2809	2809	2809	2809
	4	3303	3303	3303	3303	3303	3303	3303	3303
b'Very Good'	0	2981	2981	2981	2981	2981	2981	2981	2981
	1	2002	2002	2002	2002	2002	2002	2002	2002
	2	2531	2531	2531	2531	2531	2531	2531	2531
	3	2258	2258	2258	2258	2258	2258	2258	2258
	4	2310	2310	2310	2310	2310	2310	2310	2310

	carat	depth_percentage	table	length	width	depth	price_range
count	53299.000000	53299.000000	53299.000000	53299.000000	53299.000000	53299.000000	53299.000000
mean	0.796621	61.747412	57.455065	5.729599	5.733324	3.538592	1.871029
std	0.472033	1.425659	2.230042	1.116808	1.138125	0.700867	1.433017
min	0.200000	43.000000	43.000000	3.730000	3.680000	1.070000	0.000000
25%	0.400000	61.000000	56.000000	4.710000	4.720000	2.910000	1.000000
50%	0.700000	61.800000	57.000000	5.700000	5.710000	3.520000	2.000000
75%	1.040000	62.500000	59.000000	6.540000	6.530000	4.030000	3.000000
max	5.010000	78.200000	95.000000	10.740000	58.900000	31.800000	4.000000

Lab 3 - Extended Logistic Regression¶

Team¶

Preparation and Overview¶

Business Understanding¶

Loading the Dataset¶

Cleaning the Dataset¶

Column Renaming¶

Checking for Null Values¶

Checking for 0 values¶

Checking for Duplicate Observations¶

Dataset Statistics¶

Color¶

Clarity¶

Cut¶

Final Dataset¶

Splitting the Dataset¶

Modeling¶

Custom Model¶

Steepest Ascent¶

Binary Stochastic Gradient Descent¶

Newton's Method¶

L2 Regularization¶

Comparing Custom Models¶

Scikit Learn Model¶

Model Comparison¶

Deployment¶

Graduate Analysis¶

BFGS Method for Multiclass Logistic Regression(Quasi-Newton Method)¶

Visualization of BFGS¶

Mean Square Error¶

	carat	cut	color	clarity	depth	table	price	'x'	'y'	'z'
0	0.23	b'Ideal'	b'E'	b'SI2'	61.5	55.0	326.0	3.95	3.98	2.43
1	0.21	b'Premium'	b'E'	b'SI1'	59.8	61.0	326.0	3.89	3.84	2.31
2	0.23	b'Good'	b'E'	b'VS1'	56.9	65.0	327.0	4.05	4.07	2.31
3	0.29	b'Premium'	b'I'	b'VS2'	62.4	58.0	334.0	4.20	4.23	2.63
4	0.31	b'Good'	b'J'	b'SI2'	63.3	58.0	335.0	4.34	4.35	2.75

	carat	cut	color	clarity	depth_percentage	table	width	price_range
11182	1.07	b'Ideal'	b'F'	b'SI2'	61.6	56.0	6.62	3
11963	1.00	b'Very Good'	b'H'	b'VS2'	63.3	53.0	0.00	3
15951	1.14	b'Fair'	b'G'	b'VS1'	57.5	67.0	0.00	3
24520	1.56	b'Ideal'	b'G'	b'VS2'	62.2	54.0	0.00	4
26243	1.20	b'Premium'	b'D'	b'VVS1'	62.1	59.0	0.00	4
27429	2.25	b'Premium'	b'H'	b'SI2'	62.8	59.0	0.00	4
49556	0.71	b'Good'	b'F'	b'SI2'	64.1	60.0	0.00	2
49557	0.71	b'Good'	b'F'	b'SI2'	64.1	60.0	0.00	2

	carat	cut	color	clarity	depth_percentage	table	length	width	price_range
2207	1.00	b'Premium'	b'G'	b'SI2'	59.1	59.0	6.55	6.48	2
2314	1.01	b'Premium'	b'H'	b'I1'	58.1	59.0	6.66	6.60	2
4791	1.10	b'Premium'	b'G'	b'SI2'	63.0	59.0	6.50	6.47	2
5471	1.01	b'Premium'	b'F'	b'SI2'	59.2	58.0	6.50	6.47	2
10167	1.50	b'Good'	b'G'	b'I1'	64.0	61.0	7.15	7.04	3
11182	1.07	b'Ideal'	b'F'	b'SI2'	61.6	56.0	0.00	6.62	3
11963	1.00	b'Very Good'	b'H'	b'VS2'	63.3	53.0	0.00	0.00	3
13601	1.15	b'Ideal'	b'G'	b'VS2'	59.2	56.0	6.88	6.83	3
15951	1.14	b'Fair'	b'G'	b'VS1'	57.5	67.0	0.00	0.00	3
24394	2.18	b'Premium'	b'H'	b'SI2'	59.4	61.0	8.49	8.45	4
24520	1.56	b'Ideal'	b'G'	b'VS2'	62.2	54.0	0.00	0.00	4
26123	2.25	b'Premium'	b'I'	b'SI1'	61.3	58.0	8.52	8.42	4
26243	1.20	b'Premium'	b'D'	b'VVS1'	62.1	59.0	0.00	0.00	4
27112	2.20	b'Premium'	b'H'	b'SI1'	61.2	59.0	8.42	8.37	4
27429	2.25	b'Premium'	b'H'	b'SI2'	62.8	59.0	0.00	0.00	4
27503	2.02	b'Premium'	b'H'	b'VS2'	62.7	53.0	8.02	7.95	4
27739	2.80	b'Good'	b'G'	b'SI2'	63.8	58.0	8.90	8.85	4
49556	0.71	b'Good'	b'F'	b'SI2'	64.1	60.0	0.00	0.00	2
49557	0.71	b'Good'	b'F'	b'SI2'	64.1	60.0	0.00	0.00	2
51506	1.12	b'Premium'	b'G'	b'I1'	60.4	59.0	6.71	6.67	2